package com.ruoyi.fangyuan.webmagic;

import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Controller;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.ResponseBody;
import redis.clients.jedis.JedisPool;
import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Request;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.Spider;
import us.codecraft.webmagic.processor.PageProcessor;
import us.codecraft.webmagic.scheduler.RedisScheduler;

import javax.annotation.concurrent.ThreadSafe;
import java.util.ArrayList;
import java.util.List;

/**
 * @Title: TestRepoPageProcessor
 * @Description:
 * @Auther: 龙天
 * @Version: 1.0
 * @create 2023/5/3 19:11
 */
@Controller
@ThreadSafe
public class TestRepoPageProcessor implements PageProcessor {
    private Site site = Site.me().setRetryTimes(3).setSleepTime(100);

    /**
     * 解析返回的数据page
     * @param page
     */
    @Override
    public void process(Page page) {
        //判断是否是列表页
        if(page.getUrl().regex("https://nc\\.fang\\.lianjia\\.com/loupan/pg\\d+/").match()){
            //使用正则表达式从当前页面中获取所有符合条件的链接，并将这些链接添加到待爬取队列中
            List<String> pages = page.getHtml().links().regex("https://nc\\.fang\\.lianjia\\.com/loupan/pg\\d*/").all();
            page.addTargetRequests(pages);
            //使用 XPath 选择器获取每个楼盘的名称、价格、地址、标签和在售状态，并将这些信息存储到相应的列表中
            List<String> names = page.getHtml().xpath("//div[@class='resblock-desc-wrapper']//div[1]//a/text()").all();
            List<String> prices = page.getHtml().xpath("//div[@class='resblock-desc-wrapper']//div[@class='main-price']/span[@class='number']/text()").all();
            List<String> addresses = page.getHtml().xpath("//div[@class='resblock-desc-wrapper']//div[@class='resblock-location']/a/text()").all();
            List<String> tags = page.getHtml().xpath("//div[@class='resblock-desc-wrapper']//div[@class='resblock-tag']//span[1]/text()").all();
            List<String> urls = page.getHtml().xpath("//div[@class='resblock-desc-wrapper']//a[@class='resblock-room']//@href").all();
            page.putField("roomname", names);
            page.putField("price", prices);
            page.putField("address", addresses);
            page.putField("tags", tags);
            page.putField("urls",urls);

            //详情页连接
            List<String> url = new ArrayList<>();
            for (String str : urls) {
                url.add("https://nc.fang.lianjia.com"+str);
            }
            page.addTargetRequests(url);
            //实现动态分页
            for (int i = 2; i < 20; i++) {
                Request request = new Request("https://nc.fang.lianjia.com/loupan/pg" + i);
                page.addTargetRequest(request);
            }

        }else if (page.getUrl().regex("https://nc\\.fang\\.lianjia\\.com/loupan/p_.*").match()) {
            page.putField("name",page.getHtml().xpath("//h2[@class='DATA-PROJECT-NAME']/text()").toString());
            page.putField("otherName",page.getHtml().xpath("//div[@class='other-name']/text()").toString());
        }





    }

    @Override
    public Site getSite() {
        return site;
    }


    @Autowired
    TestPipeline testPipeline;
    @Autowired
    JedisPool jedisPool;



    @ResponseBody
    @GetMapping("/webmagic")
    public void webmagic() {
        Spider.create(new TestRepoPageProcessor())
                .addUrl("https://nc.fang.lianjia.com/loupan/")
                //防止重复爬取相同信息
                .setScheduler(new RedisScheduler(jedisPool))
                .addPipeline(testPipeline)
                //开启5个线程抓取
//                .addPipeline(testPipeline)
                .thread(5)
                //启动爬虫
                .run();
    }

    @Scheduled(cron = "0 0 12 * * ? ")
    public void scheduleSpider(){
        System.out.println("======定时爬取信息开始======");
        webmagic();
    }
}
